# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import requests
import json
import logging

from scrapy.exceptions import IgnoreRequest

class CookiesMiddleware():
    def  __init__(self, cookies_pool_url):
        self.logger = logging.getLogger(__name__)
        self.cookies_pool_url = cookies_pool_url

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            cookies_pool_url = crawler.settings.get('COOKIES_POOL_URL')
        )

    def _get_random_cookies(self):
        try:
            response = requests.get(self.cookies_pool_url)
            if response.status_code == 200:
                return json.loads(response.text)
        except ConnectionError:
            return None

    def process_request(self, request, spider):
        cookies = self._get_random_cookies()
        if cookies:
            request.cookies = cookies
            self.logger.debug('Using Cookies' + json.dumps(cookies))
        else:
            self.logger.debug('No Valid Cookies')

    # 微博的反爬虫非常厉害，可能会跳转到封号的页面当中去
    def process_response(self, request, response, spider):
        if response.status in [300, 301, 302, 303]:
            try:
                redirect_url = response.headers['location']
                # 跳转到登录页面，cookies失效了
                if 'passport' in redirect_url:
                    self.logger.warning('Need Login, Updating Cookies')
                # 若账号被封号
                elif 'weibo.cn/security' in redirect_url:
                    self.logger.warning('Account is Locked!')
                # 重新请求
                request.cookies = self._get_random_cookies()
                return request

            except:
                raise IgnoreRequest
        # （1）url请求过长
        # （2）同一个IP地址频繁访问
        elif response.status in [414]:
            return request
        else:
            return response

"""
注意1：微博反爬措施之一
    检测到同一IP请求量过大时就会出现414状态码，如果遇到这样的情况可以切换代理（需要学习代理池）
"""

